{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scanpy as sc\n",
    "import anndata as ad\n",
    "from pathlib import Path\n",
    "import glob\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os\n",
    "os.chdir('/lustre/scratch/kiviaho/prostate_spatial/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Dong et al. 2020 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc_files = glob.glob('sc-reference/dong_2020/*txt') \n",
    "dong_annot = pd.read_csv('./sc-reference/dong_2020/dong_2020_annot.csv',sep=';',index_col=0)\n",
    "dong_annot = dong_annot.rename(columns={'cells':'celltype_orig'})\n",
    "\n",
    "# Download the files into a list and concatenate together\n",
    "adata_list = []\n",
    "for file in sc_files:\n",
    "    s_abbr = '_'.join(file.split('/')[2].split('_')[0:2])\n",
    "    \n",
    "    with open(file) as x:\n",
    "        ncols = len(x.readline().split('\\t'))\n",
    "\n",
    "    df = pd.read_csv(file, usecols=range(1,ncols),delimiter='\\t',index_col=0)\n",
    "    adata = ad.AnnData(df).T\n",
    "\n",
    "    #### ADDING METADATA ####\n",
    "    adata.obs_names = s_abbr + '_' + adata.obs_names\n",
    "    meta = adata.obs.copy()\n",
    "    meta['sample'] = s_abbr\n",
    "    meta['patient'] = s_abbr\n",
    "    meta = meta.merge(dong_annot,how='left',left_index=True,right_index=True)\n",
    "    meta['phenotype'] = 'CRPC'\n",
    "    meta['dataset'] = 'dong_2020'\n",
    "\n",
    "    adata.obs = meta.copy()\n",
    "    ##########\n",
    "    adata.obs_names_make_unique()\n",
    "\n",
    "    # Since the genes were originally named with ENSEMBL ID, we have to make them unique.\n",
    "    adata.var_names_make_unique()\n",
    "    adata_list.append(adata)\n",
    "    \n",
    "adata_concat = ad.concat(adata_list, join='outer', fill_value=0)\n",
    "\n",
    "adata_concat.obs\n",
    "adata_concat.write('sc-reference/dong_2020/adata_obj.h5ad')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Chen et al. 2021 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "adata = sc.read_csv('sc-reference/chen_2021/GSM4203181_data.raw.matrix.txt',delimiter='\\t')\n",
    "adata\n",
    "adata = adata.T\n",
    "\n",
    "chen_obs = adata.obs\n",
    "#### ADDING METADATA ####\n",
    "\n",
    "chen_obs['sample'] = ['chen_'+s.split('-')[1] for s in chen_obs.index]\n",
    "chen_obs['patient'] = chen_obs['sample']\n",
    "chen_obs['celltype_orig'] = 'unknown'\n",
    "chen_obs['phenotype'] = 'PCa'\n",
    "chen_obs['dataset'] = 'chen_2021'\n",
    "\n",
    "##########\n",
    "\n",
    "\n",
    "if (chen_obs.index == adata.obs_names).all():\n",
    "    adata.obs = chen_obs\n",
    "adata.obs_names = adata.obs['sample'] + '_' + [s.split('-')[0] for s in adata.obs_names] + '.1'\n",
    "adata.obs.index = adata.obs.index.set_names(['cell'])\n",
    "adata.obs\n",
    "\n",
    "adata.write('sc-reference/chen_2021/adata_obj.h5ad')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Song et al. 2022 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This information is from supplementary file 1 of the article Song et al. 2022 Nature Comms\n",
    "# The cell type annotations are available without cell IDs, so merging isn't possible.\n",
    "# Number of analysed cells is 21743\n",
    "song_samples = ['AUG_PB1A', 'AUG_PB1B','MAY_PB1A','MAY_PB1B', 'MAY_PB2A','MAY_PB2B', # BIOPSIES\n",
    "                'PR5186','PR5196','PR5199','PR5269', # UNPAIRED RPs\n",
    "                'PR5249_N','PR5249_T', # NORMAL PAIRED RPs\n",
    "                'PR5251_N','PR5251_T', # NORMAL PAIRED RPs\n",
    "                'PR5254_N','PR5254_T', # NORMAL PAIRED RPs\n",
    "                'PR5261_N','PR5261_T'] # NORMAL PAIRED RPs\n",
    "\n",
    "song_patients = ['P1','P1','P2','P2','P3','P3',\n",
    "                 'P4','P5','P6','P7',\n",
    "                 'P8','P8',\n",
    "                 'P9','P9',\n",
    "                 'P10','P10',\n",
    "                 'P11','P11']\n",
    "song_phenotype = list(np.repeat('PCa',10)) + ['normal','PCa','normal','PCa','normal','PCa','normal','PCa']\n",
    "\n",
    "\n",
    "# Replace some of the idents with those matching file names\n",
    "song_file_names = song_samples.copy()\n",
    "song_file_names[:] = ['AUG_PB_1A' if x=='AUG_PB1A' else x for x in song_file_names]\n",
    "song_file_names[:] = ['AUG_PB_1B' if x=='AUG_PB1B' else x for x in song_file_names]\n",
    "\n",
    "song_file_names[:] = ['PB1A' if x=='MAY_PB1A' else x for x in song_file_names]\n",
    "song_file_names[:] = ['PB1B' if x=='MAY_PB1B' else x for x in song_file_names]\n",
    "\n",
    "song_file_names[:] = ['PB2A' if x=='MAY_PB2A' else x for x in song_file_names]\n",
    "song_file_names[:] = ['PB2B' if x=='MAY_PB2B' else x for x in song_file_names]\n",
    "\n",
    "\n",
    "adata_samples_list = []\n",
    "for idx,file_abbr in enumerate(song_file_names):\n",
    "\n",
    "    # Find all files generated from one sample\n",
    "    file_name_list = glob.glob('sc-reference/song_2022/*'+file_abbr+'*')\n",
    "    sample_abbr = song_samples[idx]\n",
    "    patient_abbr = song_patients[idx]\n",
    "    phenot_abbr = song_phenotype[idx]\n",
    "\n",
    "    sample_adata = []\n",
    "\n",
    "    # Read in each file and append them to a sample-specific list\n",
    "    for f in file_name_list:\n",
    "        # exp_abbr = f.split('/')[2].split('_')[0]\n",
    "        adata = sc.read_csv(f,dtype=np.int16,delimiter='\\t').T\n",
    "        adata.obs_names = sample_abbr +'_'+ adata.obs_names + '-1'\n",
    "        sample_adata.append(adata)\n",
    "    adata_concat_one_sample = ad.concat(sample_adata, join='outer', fill_value=0)\n",
    "\n",
    "    # Concatenate together data from the same sample but from different sequencing runs.\n",
    "    adata_concat_one_sample.obs['sample'] = sample_abbr\n",
    "    adata_concat_one_sample.obs['patient'] = 'song_'+patient_abbr\n",
    "    adata_concat_one_sample.obs['celltype_orig'] = 'unknown'\n",
    "    adata_concat_one_sample.obs['phenotype'] = phenot_abbr\n",
    "    adata_concat_one_sample.obs['dataset'] = 'song_2022'\n",
    "    adata_samples_list.append(adata_concat_one_sample)\n",
    "\n",
    "adata_concat_all = ad.concat(adata_samples_list, join='outer', fill_value=0)\n",
    "adata_concat_all.obs\n",
    "\n",
    "adata_concat_all.write('sc-reference/song_2022/adata_obj.h5ad')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Cheng et al. 2022 data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dirs = glob.glob('sc-reference/cheng_2022/results/*')\n",
    "new_names = pd.read_csv('sc-reference/cheng_2022/sample_shorthands.txt',sep='\\t')\n",
    "\n",
    "adata_list = []\n",
    "for dir in data_dirs:\n",
    "    # Get the shorhand\n",
    "    sample = dir.split('/')[-1]\n",
    "    shorthand = new_names[new_names['old']==sample]['new'].item()\n",
    "    patient = shorthand.split('_')[0]\n",
    "    if 'CRPC' in patient:\n",
    "        phenot_abbr = 'CRPC'\n",
    "    else:\n",
    "        phenot_abbr = 'PCa'\n",
    "\n",
    "    adata = sc.read_10x_mtx(dir+'/outs/filtered_feature_bc_matrix')\n",
    "\n",
    "    adata.obs['sample'] = shorthand\n",
    "    adata.obs['patient'] = 'cheng_'+patient\n",
    "    adata.obs['celltype_orig'] = 'unknown'\n",
    "    adata.obs['phenotype'] = phenot_abbr\n",
    "    adata.obs['dataset'] = 'cheng_2022'\n",
    "\n",
    "    adata.obs_names = shorthand + '_' + adata.obs_names\n",
    "    adata_list.append(adata)\n",
    "\n",
    "adata_concat = ad.concat(adata_list, join='outer', fill_value=0)\n",
    "adata_concat.obs\n",
    "adata_concat.write('sc-reference/cheng_2022/adata_obj.h5ad')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Wong et al. 2022 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat = sc.read_h5ad('sc-reference/wong_2022/wong_2022_data.h5ad')\n",
    "annot = dat.obs.copy()\n",
    "cell_annot = pd.read_csv('sc-reference/wong_2022/GSE185344_PH_scRNA.rename_cluster.csv')\n",
    "cell_annot.index = cell_annot['Unnamed: 0']\n",
    "merged_obs = pd.merge(annot,cell_annot,left_index=True,right_index=True,how='left')\n",
    "\n",
    "if (merged_obs.index == dat.obs.index).all():\n",
    "        dat.obs = merged_obs\n",
    "        del dat.raw\n",
    "\n",
    "# Format the phenotypes\n",
    "phenot = [s.split('_')[2] for s in dat.obs['orig.ident']]\n",
    "phenot = [w.replace('Benign', 'normal') for w in phenot]\n",
    "phenot = [w.replace('Tumor', 'PCa') for w in phenot]\n",
    "\n",
    "# Format the patient \n",
    "patient = ['_'.join(s.split('_')[:2]) for s in dat.obs['orig.ident']]\n",
    "\n",
    "new_obs = pd.DataFrame()\n",
    "new_obs.index = dat.obs.index.copy()\n",
    "new_obs['sample'] = 'wong2022_'+dat.obs['orig.ident'].copy()\n",
    "new_obs['patient'] = ['wong2022_'+ p for p in patient]\n",
    "new_obs['celltype_orig'] = dat.obs['cellactivity_clusters'].copy()\n",
    "new_obs['phenotype'] = phenot\n",
    "new_obs['dataset'] = 'wong_2022'\n",
    "\n",
    "dat.obs = new_obs\n",
    "\n",
    "# Lose the unnecessary column 'features'\n",
    "dat.var = dat.var.drop(columns='features')\n",
    "\n",
    "dat.write('sc-reference/wong_2022/adata_obj.h5ad')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Chen et al. 2022 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dirs = glob.glob('sc-reference/chen_2022/results/*')\n",
    "\n",
    "adata_list = []\n",
    "for dir in data_dirs:\n",
    "    # Get the shorhand\n",
    "    print(dir)\n",
    "    sample = dir.split('/')[-1]\n",
    "    if 'PCa' in sample:\n",
    "        phenot_abbr = 'PCa'\n",
    "    else:\n",
    "        phenot_abbr = 'normal'\n",
    "\n",
    "    adata = sc.read_10x_mtx(dir+'/outs/filtered_feature_bc_matrix')\n",
    "\n",
    "    adata.obs['sample'] = 'chen2022_'+sample\n",
    "    adata.obs['patient'] = 'chen2022_'+sample\n",
    "    adata.obs['celltype_orig'] = 'unknown'\n",
    "    adata.obs['phenotype'] = phenot_abbr\n",
    "    adata.obs['dataset'] = 'chen_2022'\n",
    "\n",
    "    adata.obs_names = sample + '_' + adata.obs_names\n",
    "    adata_list.append(adata)\n",
    "\n",
    "adata_concat = ad.concat(adata_list, join='outer', fill_value=0)\n",
    "adata_concat.obs\n",
    "adata_concat.write('sc-reference/chen_2022/adata_obj.h5ad')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting Hirz et al. 2023 data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Download the files into a list and concatenate together\n",
    "# There are no counts for GSM5494349_SCG-PCA2-T-LG.count.csv\n",
    "# PCA24 samples have been prepared another way, excluded\n",
    "\n",
    "sc_files = sorted(glob.glob('sc-reference/hirz_2023/*SCG*')) # adjacent normal tissue\n",
    "print(sc_files)\n",
    "print()\n",
    "hirz_annot = pd.read_csv('sc-reference/hirz_2023/GSE181294_scRNAseq.ano.csv',index_col=0)\n",
    "hirz_annot = hirz_annot.drop(columns=['sample'])\n",
    "hirz_annot = hirz_annot.rename(columns={'cells':'celltype_orig'})\n",
    "\n",
    "adata_list = []\n",
    "for f in sc_files:\n",
    "    name_split = f.split('/')[-1].split('_')[-1].split('-')\n",
    "    sample = ('_').join(name_split[1:3])\n",
    "    patient = name_split[1]\n",
    "    if name_split[2] == 'N':\n",
    "        phenot_abbr = 'normal'\n",
    "    else:\n",
    "        phenot_abbr = 'PCa'\n",
    "    \n",
    "    adata = sc.read_csv(f,dtype=np.int16)\n",
    "    adata = adata.T\n",
    "\n",
    "    ####### Add metadata columns\n",
    "    meta = adata.obs.copy()\n",
    "    meta['sample'] = 'hirz_'+sample\n",
    "    meta['patient'] = 'hirz_'+patient\n",
    "    meta = meta.merge(hirz_annot,how='left',left_index=True,right_index=True,)\n",
    "    meta['phenotype'] = phenot_abbr\n",
    "    meta['dataset'] = 'hirz_2023'\n",
    "\n",
    "    if (meta.index == adata.obs_names).all():\n",
    "        adata.obs = meta.copy()\n",
    "    print(sample)\n",
    "    adata_list.append(adata)\n",
    "\n",
    "adata_concat = ad.concat(adata_list, join='outer', fill_value=0)\n",
    "\n",
    "adata_concat.obs\n",
    "adata_concat.write('sc-reference/hirz_2023/adata_obj.h5ad')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
